package com.shiguiwu.mogu.spider;

import cn.hutool.core.util.RandomUtil;
import com.shiguiwu.mogu.entity.Banner;
import com.shiguiwu.mogu.service.BannerService;
import com.shiguiwu.mogu.service.impl.BannerServiceImpl;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.PlainText;

import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.List;

/**
 * @description: git爬虫
 * @author: stone
 * @date: Created by 2020/1/2 20:59
 * @version: 1.0.0
 * @pakeage: com.shiguiwu.spider
 */
@Component
public class GithubRepoPageProcessor implements PageProcessor {

    private Site site = Site.me().setRetryTimes(3).setSleepTime(100);
    public static String url = "https://search.jd.com/Search?keyword=%E5%A5%B3%E8%A3%85&wq=%E5%A5%B3%E8%A3%85&page=6&s=56";

    //    @Resource
    private BannerService bannerService = new BannerServiceImpl();

    @Override
    public void process(Page page) {

//        page.addTargetRequests(page.getHtml().links().regex("(https://m.mogu\\\\.com/\\\\w+/\\\\w+)").all());
//        page.putField("main", page.getHtml().regex(".*App.*").toString());gl-i-wrap  p-img
//        page.putField("name", page.getHtml().xpath("//*[@class=\"gl-warp clearfix\"]//[@class=\"gl-item\"]//[@class=\"gl-i-wrap\"]"));
        page.putField("link", page.getHtml().xpath("//*[@class=\"gl-warp clearfix\"]//[@class=\"gl-item\"]//[@class=\"gl-i-wrap\"]//[@class=\"p-img\"]//a/@href"));
        page.putField("image", page.getHtml().xpath("//*[@class=\"gl-warp clearfix\"]//[@class=\"gl-item\"]//[@class=\"gl-i-wrap\"]//[@class=\"p-img\"]//a/img/@data-lazy-img"));
        page.putField("title", page.getHtml().xpath("//*[@class=\"gl-warp clearfix\"]//[@class=\"gl-item\"]//[@class=\"gl-i-wrap\"]//[@class=\"p-name p-name-type-2\"]//em/text()"));
        page.putField("price", page.getHtml().xpath("//*[@class=\"gl-warp clearfix\"]//[@class=\"gl-item\"]//[@class=\"gl-i-wrap\"]//[@class=\"p-price\"]//strong//i/text()"));
//        page.putField("comment", page.getHtml().xpath("//*[@class=\"gl-warp clearfix\"]//[@class=\"gl-item\"]//[@class=\"gl-i-wrap\"]//[@class=\"p-commit\"]//strong//a"));

        Object o = page.getResultItems().get("link");
        Object o1= page.getResultItems().get("image");
        Object o2 = page.getResultItems().get("title");
        Object o3 = page.getResultItems().get("price");
//        Object o4 = page.getResultItems().get("comment");
        List<String> sourceTexts = ((PlainText) o).all();
        List<String> sourceTexts1 = ((PlainText) o1).all();
        List<String> sourceTexts2 = ((PlainText) o2).all();
        List<String> sourceTexts3 = ((PlainText) o3).all();
        List<Banner> list = new ArrayList<>();
        for (int i = 0; i < sourceTexts.size(); i++) {
            Banner banner = new Banner();
            banner.setLink("http:"+sourceTexts.get(i)).setImage("http:" + sourceTexts1.get(i))
                    .setTitle(sourceTexts2.get(i)).setPrice(sourceTexts3.get(i)).setComment(RandomUtil.randomInt(10000, 100000) + "");
            list.add(banner);

        }

        bannerService.saveBatch(list);




    }

    @Override
    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {


    }

    public void execute() {
        Spider.create(new GithubRepoPageProcessor())
                .addUrl(url)
//                .addPipeline(new JsonFilePipeline("G:\\webmagic\\"))
                .thread(5).run();
    }
}
